import os
import random
import time
import pymysql_tool as tool

import requests
from lxml import etree


def get_one_page_data(page,t_name,connect,curs):
    li_list = page.xpath('//ul[@class="viewlist_ul"]/li')
    for li in li_list:
        # 汽车名字
        car_name = li.xpath('./a/div[@class="cards-bottom"]/h4/text()')
        if len(list(car_name)) == 0:
            continue
        car_name = list(car_name)[0]

        # 汽车上牌信息
        car_info = str(list(li.xpath('./a/div[@class="cards-bottom"]/p/text()'))[0])
        car_info_list = car_info.split("／")
        gong_li = car_info_list[0]
        card_time = car_info_list[1]
        address = car_info_list[2]
        hui_yuan_years = car_info_list[3]
        # print(car_name,gong_li,card_time,address,hui_yuan_years)

        # 汽车价格
        price_list = list(li.xpath('./a/div[@class="cards-bottom"]/div[@class="cards-price-box"]/s/text()'))
        if len(price_list) == 0:
            car_price = '无'
        else:
            car_price = str(price_list[0])

        car_sell_price_list = list(
            li.xpath('./a/div[@class="cards-bottom"]/div[@class="cards-price-box"]/span[@class="pirce"]/em/text()'))
        if len(car_sell_price_list) == 0:
            car_sell_price = '无'
        else:
            car_sell_price = str(car_sell_price_list[0])
        # print(car_name, gong_li, card_time, address, hui_yuan_years,car_price,car_sell_price)

        # 标签
        tags_i = li.xpath('./a/div[@class="cards-bottom"]/div[@class="cards-price-box"]/span[@class="tags"]/i')
        tags_list = []
        if len(tags_i) == 0:
            tags = '无'
        else:
            for i in tags_i:
                tags_list.append(str(i.xpath('string(.)')))
            tags = '|'.join(tags_list)

        curs.execute(f"insert into {t_name}(car_name, gong_li, card_time, address, hui_yuan_years, car_price, car_sell_price, tags) values(%s,%s,%s,%s,%s,%s,%s,%s)",[car_name, gong_li, card_time, address, hui_yuan_years, car_price, car_sell_price, tags])
        connect.commit()
        print(car_name, gong_li, card_time, address, hui_yuan_years, car_price, car_sell_price, tags)
        print("-------------------")


if __name__ == '__main__':
    # conn = tool.get_connection()
    # cur = conn.cursor()
    # # https://www.che168.com/hefei/a0_0msdgscncgpi1ltocsp{i}exx0/?pvareaid=102179
    # h = {
    #     'cookie': 'fvlid=1734397465811h7jvf7DJTU0t; sessionid=1682b984-2b6b-4000-9ed7-409a1704ce7d; che_sessionid=D20B4B35-D4C1-4381-BA92-0929DFF6CCD4%7C%7C2024-12-17+09%3A04%3A26.109%7C%7Cwww.autohome.com.cn; Hm_lvt_d381ec2f88158113b9b76f14c497ed48=1734397466,1734397676; HMACCOUNT=51CFE8B0D92C936E; sessionvisit=86bec71d-2543-4f10-b7d7-b4482b006a83; sessionvisitInfo=1682b984-2b6b-4000-9ed7-409a1704ce7d||0; che_sessionvid=DED4F558-9ADD-4FF6-B1FF-3BC1F835EBAF; userarea=340100; listuserarea=340100; smidV2=20241217105808fc55c58d7bc86edcd92d058b4816dbdd00d30d442ad8d72f0; pcpopclub=241ddbe2714c4f78bec81cd65735b5ae11428def; clubUserShow=289574383|0|340100|%e5%90%88%e8%82%a5%e8%bd%a6%e5%8f%8b3267694|0|0|0|/g30/M05/0C/33/120X120_0_q87_autohomecar__CjIFU2T-ztCASsKCAACCWuPPeYk210.png|2024-12-17 10:59:20|0; sessionlogin=241ddbe2714c4f78bec81cd65735b5ae11428def; autouserid=289574383; href=https%3A%2F%2Fwww.che168.com%2F; accessId=7a783820-ec84-11ec-b95f-79694d4df285; pageViewNum=4; sessionip=154.64.226.11; area=0; ahpvno=45; Hm_lpvt_d381ec2f88158113b9b76f14c497ed48=1734408112; showNum=45; ahuuid=77955BB2-1FB6-4585-AC11-321386A417BB; sessionuid=1682b984-2b6b-4000-9ed7-409a1704ce7d; v_no=45; visit_info_ad=D20B4B35-D4C1-4381-BA92-0929DFF6CCD4||DED4F558-9ADD-4FF6-B1FF-3BC1F835EBAF||-1||-1||45; che_ref=www.autohome.com.cn%7C0%7C100533%7C0%7C2024-12-17+12%3A01%3A53.433%7C2024-12-17+09%3A04%3A26.109',
    #     'sec-ch-ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
    #     'sec-ch-ua-mobile': '?0',
    #     'sec-ch-ua-platform': '"Windows"',
    #     'sec-fetch-dest': 'document',
    #     'sec-fetch-mode': 'navigate',
    #     'sec-fetch-site': 'none',
    #     'sec-fetch-user': '?1',
    #     'upgrade-insecure-requests': '1',
    #     'priority': 'u=0, i',
    #     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
    # }
    #
    # u = f'https://www.che168.com/fu_yang/fuyang/list/#pvareaid=100943'
    # res = requests.get(url=u, headers=h)
    # res.encoding = 'gb2312'
    # car_page_html = etree.HTML(res.text)
    # # 爬取当页数据
    # get_one_page_data(car_page_html,'car_fuyang',conn,cur)

    # proxy_pools = [
    #     {'http': 'http://154.127.240.126:64003'},
    #     {'http': 'http://77.235.31.24:8080'},
    #     {'http': 'http://103.139.242.169:83'},
    #     {'http': 'http://72.10.164.178:12867'},
    #     {'http': 'http://154.127.240.126:64003'},
    #     {'http': 'http://77.235.31.24:8080'},
    #     {'http': 'http://103.139.242.169:83'},
    #     {'http': 'http://67.43.227.227:12629'},
    #     {'http': 'http://72.10.160.174:17789'},
    #     {'http': 'http://47.236.238.103:80'},
    #     {'http': 'http://46.253.143.144:3128'},
    #     {'http': 'http://38.145.211.247:8899'},
    #     {'http': 'http://111.206.0.98:8080'}
    # ]
    city_list = ['anqing','fu_yang','bangbu','bozhou','chizhou','chuzhou','hefei','huaibei','huainan',
                 'liuan','maanshan','su_zhou','tongling','wuhu','xuancheng','huangshan']

    '''
    创建数据库连接对象和操作对象
    '''

    conn = tool.get_connection()
    cur = conn.cursor()

    for city in city_list:
        cur.execute(f"truncate table `car_{city}`")
        conn.commit()

        cur.execute(f"""
        CREATE TABLE if not exists `car_{city}` (
          `id` int(11) NOT NULL AUTO_INCREMENT,
          `car_name` varchar(255) DEFAULT NULL,
          `gong_li` varchar(255) DEFAULT NULL,
          `card_time` varchar(255) DEFAULT NULL,
          `address` varchar(255) DEFAULT NULL,
          `hui_yuan_years` varchar(255) DEFAULT NULL,
          `car_price` varchar(255) DEFAULT NULL,
          `car_sell_price` varchar(255) DEFAULT NULL,
          `tags` varchar(255) DEFAULT NULL,
          PRIMARY KEY (`id`)
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8
        """)
        conn.commit()

        # time.sleep(20)
        print(f"==============================正在获取城市：{city} 的二手车信息=======================================")
        i = 1
        for i in range(1, 10):
            # proxies = random.choice(proxy_pools)
            # https://www.che168.com/hefei/a0_0msdgscncgpi1ltocsp{i}exx0/?pvareaid=102179
            h = {
                'cookie': 'fvlid=1734397465811h7jvf7DJTU0t; sessionid=1682b984-2b6b-4000-9ed7-409a1704ce7d; che_sessionid=D20B4B35-D4C1-4381-BA92-0929DFF6CCD4%7C%7C2024-12-17+09%3A04%3A26.109%7C%7Cwww.autohome.com.cn; smidV2=20241217105808fc55c58d7bc86edcd92d058b4816dbdd00d30d442ad8d72f0; pcpopclub=241ddbe2714c4f78bec81cd65735b5ae11428def; clubUserShow=289574383|0|340100|%e5%90%88%e8%82%a5%e8%bd%a6%e5%8f%8b3267694|0|0|0|/g30/M05/0C/33/120X120_0_q87_autohomecar__CjIFU2T-ztCASsKCAACCWuPPeYk210.png|2024-12-17 10:59:20|0; sessionlogin=241ddbe2714c4f78bec81cd65735b5ae11428def; autouserid=289574383; accessId=7a783820-ec84-11ec-b95f-79694d4df285; pageViewNum=4; area=319999; sessionvisit=a0098af3-edb3-4b93-bf0b-fa822f43c6d0; sessionvisitInfo=1682b984-2b6b-4000-9ed7-409a1704ce7d|www.autohome.com.cn|100533; Hm_lvt_d381ec2f88158113b9b76f14c497ed48=1734397466,1734397676,1734416792; HMACCOUNT=51CFE8B0D92C936E; che_sessionvid=6CAE8E05-CB3F-40B9-931B-0E99C1D1E6AE; userarea=340300; listuserarea=340300; ahpvno=52; sessionip=101.227.93.57; Hm_lpvt_d381ec2f88158113b9b76f14c497ed48=1734416815; ahuuid=E564D3EE-EC39-4E40-9F3F-6765E71C9C3D; showNum=52; v_no=52; visit_info_ad=D20B4B35-D4C1-4381-BA92-0929DFF6CCD4||6CAE8E05-CB3F-40B9-931B-0E99C1D1E6AE||-1||-1||52; che_ref=www.autohome.com.cn%7C0%7C100533%7C0%7C2024-12-17+14%3A26%3A55.229%7C2024-12-17+09%3A04%3A26.109; sessionuid=1682b984-2b6b-4000-9ed7-409a1704ce7d',
                'sec-ch-ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
                'sec-ch-ua-mobile': '?0',
                'sec-ch-ua-platform': '"Windows"',
                'sec-fetch-dest': 'document',
                'sec-fetch-mode': 'navigate',
                'sec-fetch-site': 'none',
                'sec-fetch-user': '?1',
                'upgrade-insecure-requests': '1',
                'priority': 'u=0, i',
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
            }
            print(
                f"--------------------------正在获取城市：{city} 的二手车第 {i} 页信息----------------------------------------------------------------------------")
            u = f'https://www.che168.com/{city}/a0_0msdgscncgpi1ltocsp{i}exx0/?pvareaid=102179'
            res = requests.get(url=u, headers=h)
            res.encoding = 'gb2312'
            car_page_html = etree.HTML(res.text)

            table_name = f"car_{city}"
            # 爬取当页数据
            get_one_page_data(car_page_html,table_name,conn,cur)

            tmp_list = car_page_html.xpath('//div[@id="listpagination"]/a[last()]')
            np = None
            if len(list(tmp_list)) == 0:
                while True:
                    time.sleep(1.2)
                    try:
                        print(f"重试：{u}")
                        res = requests.get(url=u, headers=h)
                        res.encoding = 'gb2312'
                        car_page_html = etree.HTML(res.text)
                        np = car_page_html.xpath('//div[@id="listpagination"]/a[last()]')[0].attrib.get('class')
                        break
                    except:
                        continue
            if np == 'page-item-next page-disabled':
                break
            i += 1
